from pathlib import Path
import random
import time
import re

from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import StaleElementReferenceException
from selenium.common.exceptions import ElementClickInterceptedException
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import ElementNotInteractableException
from bs4 import BeautifulSoup
import gspread

from browser import Browser
import proxy_with_authorization


class Parser(Browser):

    """ Этот класс представляет парсер """

    def __init__(self, data, proxy_list=None):

        self.display_browser = data[0]
        self.where_to_start = data[2] - 1
        self.proxy_list = proxy_list
        self.number_branches_to_pass_through = 1
        self.answer_number_select = 0
        self.city_number = data[3]
        self.test_number = data[4]
        self.string_to_write = data[5]


    def wait(self, f=2, b=3.5):

        """ Чуть-чуть подождать """

        number = random.uniform(f, b)
        print(f'Ждем {number}\n')
        time.sleep(number)


    def get_saved_page(self, file_name):

        """ Открыть сохраненную страницу для парсинга и возвращаем ее """

        folder = Path('saved_html/')
        page = folder/file_name

        with open(page, 'r', encoding='utf-8') as file:

            text = file.read()
            page = BeautifulSoup(text, 'lxml')

            return page


    def convert_to_bs(self, browser):

        """ Получить объект BeautifulSoup """

        html_source = browser.page_source
        page = BeautifulSoup(html_source, 'lxml')

        return page


    def click_on_button(self, element_selenium, a=.5, b=.8):

        """ Нажать на кнопку """

        element_selenium.click()    # Кликаем на элемент
        self.wait(a, b)             # Чуть-чуть ждем


    def enter_text_in_field(self, element_selenium, text, a=.5, b=.8):

        """ Ввести текст в поле """

        element_selenium.click()
        element_selenium.clear()
        element_selenium.send_keys(text)
        self.wait(a, b)


    def move_to_element(self, browser, selenium_element, a=.5, b=.8):

        """ Переместится к элементу страницы """

        action = ActionChains(browser)
        action.move_to_element(selenium_element)
        action.perform()
        self.wait(a, b)


    def scroll_the_page_js(self, browser, a=.2, b=.5, w=0, h=50):

        """
        Переместиться по странице js
        w - прокрутить страницу по ширине
        h - прокрутить страницу по высоте
        """

        browser.execute_script(f"window.scrollBy({w}, {h})")
        self.wait(a, b)


    def get_configured_browser(self, through_proxy=None, proxy=None):

        """ Получить настроенный браузер """

        if through_proxy == 'public_proxy':  # Если необходимо делать запрос через общедоступные прокси

            browser = self.get_browser(self.display_browser, through_proxy)

            return browser

        elif through_proxy == 'private_proxy':  # Если необходимо делать запрос через приватные прокси

            def get_information_for_proxy(proxy):

                pass


            # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

            host, port, username, password = get_information_for_proxy(proxy)
            browser = proxy_with_authorization.proxy_chrome(host, port, username, password)

            return browser

        else:  # Если прокси не используюется

            browser = self.get_browser(self.display_browser, through_proxy)

            return browser


    def open_page(self, link, fs='no'):

        """
        Открываем страницу
        link - ссылка на страницу
        fs - открыть браузер в полном экране?
        """

        browser = self.get_configured_browser()
        browser.get(link)

        if fs == 'yes': browser.maximize_window()

        return browser


    def get_list_of_cities(self):

        """ Получаем ссылки на города """

        result = []
        data_folder = Path('cities/')
        file_to_open = data_folder/'list_of_cities.txt'

        with open(file_to_open) as file:

            for line in file:

                city = line.strip()
                result.append(city)

        return result


    def get_question_name(self, browser):

        """ Получаем название вопроса """

        while True:

            try:

                question_name_selenium = browser.find_element(
                    By.XPATH,
                    '//form//h3'
                )

                return question_name_selenium.text

            except StaleElementReferenceException as error:

                print(error)
                break

            except NoSuchElementException as error:

                print('Название вопроса не найдено')
                return 'Пусто'


    def click_continue(self, browser):

        """ Нажимаем продолжить """

        count_escape = 0

        while True:

            try:

                continue_button = browser.find_element(
                    By.XPATH,
                    '//span[contains(text(), "Продолжить")]/..'
                )
                self.click_on_button(continue_button)

                break

            except ElementClickInterceptedException as error:

                if count_escape == 3:

                    break

                ActionChains(browser).key_down(Keys.ESCAPE).key_up(Keys.ESCAPE).perform()
                count_escape += 1
                time.sleep(.5)


    def delete_text_from_field(self, selenium_element):

        """ Удаляем текст из поля для ввода текста """

        selenium_element.click()
        selenium_element.send_keys(Keys.CONTROL + 'a')
        selenium_element.send_keys(Keys.DELETE)


    def get_content_question_selenium(self, browser, question_name):

        """ Получаем содержимое вопроса в виде selenium """

        def get_answers_to_choose_from(browser):

            """ Получаем ответы для выбора """

            def get_only_necessary_ones(answers):

                """ Получаем только нужные """

                only_necessary_ones = []

                for answer in answers:

                    try:

                        if answer.text: only_necessary_ones.append(answer)

                    except StaleElementReferenceException as error:

                        print('Элемнет устарел')
                        print(error)

                        return []

                return only_necessary_ones


            # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

            answers_to_choose_from_selenium = browser.find_elements(
                By.XPATH,
                '//form/div/div/div/label'
            )
            only_necessary_ones = get_only_necessary_ones(answers_to_choose_from_selenium)

            return only_necessary_ones


        def get_answers_for_entering_usual(browser, question_name):

            """ Получаем ответы для ввода обычные """

            if question_name == 'Ваш адрес': return []
            if 'Куда вам удобно приехать' in question_name: return []

            answer_options = browser.find_elements(
                By.XPATH,
                '//div[contains(text(), "Например:")]/following::div[1]/div'
            )
            if answer_options: return []

            only_necessary_ones = []

            answers_to_enter_selenium = browser.find_elements(
                By.XPATH,
                '//form//input[@type="text"]'
            )

            for answer in answers_to_enter_selenium:

                try:

                    if not answer.get_attribute('readonly'):

                        only_necessary_ones.append(answer)

                except StaleElementReferenceException:

                    continue

            return only_necessary_ones


        def get_answers_for_entering_numbers(browser):

            """ Получаем ответ для ввода чисел """

            answers_for_entering_numbers = browser.find_elements(
                By.XPATH,
                '//form//input[@type="number"]'
            )

            return answers_for_entering_numbers


        def get_answers_for_entering_large_text(browser):

            """ Получаем ответы для ввода большого текста """

            answers_for_entering_large_text = browser.find_elements(
                By.XPATH,
                '//textarea[@data-shmid="textarea"]'
            )

            return answers_for_entering_large_text


        def get_answers_options(browser):

            """ Получаем варианты ответов """

            answer_options = browser.find_elements(
                By.XPATH,
                '//div[contains(text(), "Например:")]/following::div[1]/div'
            )
            return answer_options


        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

        answers_to_choose_from = get_answers_to_choose_from(browser)
        answers_for_entering_usual = get_answers_for_entering_usual(browser, question_name)
        answers_for_entering_numbers = get_answers_for_entering_numbers(browser)
        answers_for_entering_large_text = get_answers_for_entering_large_text(browser)
        answers_options = get_answers_options(browser)

        content = {
            'Ответы для выбора': [answers_to_choose_from, len(answers_to_choose_from)],
            'Ответы для ввода обычные': [answers_for_entering_usual, len(answers_for_entering_usual)],
            'Ответы для ввода чисел': [answers_for_entering_numbers, len(answers_for_entering_numbers)],
            'Ответы для ввода большого текста': [answers_for_entering_large_text, len(answers_for_entering_large_text)],
            'Ответы для выбора два': [answers_options, len(answers_options)],
        }

        return content


    def answer_specific_question(self, browser, well_what_map, well_what_calendar):

        """ Отвечаем на определенные вопросы """

        def work_with_specifying_address(browser):

            """ Работаем с указанием адреса """

            address_input_field = browser.find_element(
                By.XPATH,
                '//input[@placeholder="Город, улица и дом"]'
            )
            self.delete_text_from_field(address_input_field)
            address_input_field.send_keys('Москва, Никольская улица, 5/1с1')
            time.sleep(3.5)
            address_input_field.send_keys(Keys.DOWN)
            address_input_field.send_keys(Keys.ENTER)


        def work_with_date_indication(browser):

            """ Работаем с указанием даты """

            def select_date_in_calendar(number_to_choose_from, calendar_number):

                """ Выбираем номер в календаре """

                pass


            def get_numbers_to_choose_from(browser):

                """ Получаем числа для выбора """

                list_numbers_to_choose_from_selenium = browser.find_elements(
                    By.XPATH,
                    '//table/tbody/tr[last()]/td[last()-3]'
                )

                return list_numbers_to_choose_from_selenium


            def open_calendar(browser, current_iteration):

                """ Открываем календарь """

                calendar_selenium = browser.find_elements(
                    By.XPATH,
                    '//input[@placeholder="Дата"]'
                )[current_iteration]
                self.click_on_button(calendar_selenium)


            # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

            calendar_number = 0
            numbers_to_choose_from = get_numbers_to_choose_from(browser)

            for current_iteration, number_to_choose_from in enumerate(numbers_to_choose_from):

                while True:

                    try:

                        #select_date_in_calendar(number_to_choose_from, calendar_number)
                        number_to_choose_from.click()
                        break

                    except ElementNotInteractableException as error:

                        print(error)
                        print('Открываем календарь')
                        open_calendar(browser, current_iteration)

                    except ElementClickInterceptedException as error:

                        print('Скролим ниже')
                        self.scroll_the_page_js(browser, h=50)


                time.sleep(1)


        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

        if well_what_map: work_with_specifying_address(browser)
        if well_what_calendar: work_with_date_indication(browser)


    def answer_question(self, browser, content_question_selenium):

        """ Отвечаем на вопрос """

        def work_with_answers_to_choose_from(browser, answers):

            answer = answers[self.answer_number_select]
            self.scroll_the_page_js(browser, h=-50000)

            while True:

                try:

                    self.move_to_element(browser, answer)
                    self.scroll_the_page_js(browser, h=50)
                    self.click_on_button(answer)

                    break

                except StaleElementReferenceException as error:

                    print(error)
                    break

                except ElementClickInterceptedException as error:

                    print('Скролим ниже')
                    self.scroll_the_page_js(browser, h=50)


        def work_with_answers_for_entering_usual(browser, answers):

            """ Работаем с ответами для ввода обычными """

            for answer in answers:

                while True:

                    try:

                        self.move_to_element(browser, answer)
                        self.scroll_the_page_js(browser)
                        self.delete_text_from_field(answer)
                        self.enter_text_in_field(answer, 100)

                        break

                    except ElementClickInterceptedException as error:

                        print('Элемент перекрыт, прокручиваем вниз')
                        self.scroll_the_page_js(browser, h=50)

                    except StaleElementReferenceException as error:

                        print('Попался старый элемент, он не нужен')
                        break


        def work_with_answers_for_entering_number(browser, answers):

            """ Работаем с ответами для ввода чисел """

            for answer in answers:

                try:

                    self.move_to_element(browser, answer)
                    self.scroll_the_page_js(browser)
                    self.enter_text_in_field(answer, 10)

                except StaleElementReferenceException as error:

                    print('Элемент с другой страницы')
                    print(error)
                    continue


        def work_with_answers_to_choose_from_two(browser, answers):

            """ Работаем с предложенными ответами """

            answer = answers[0]

            while True:

                try:

                    self.move_to_element(browser, answer)
                    self.scroll_the_page_js(browser)
                    self.click_on_button(answer)

                    break

                except StaleElementReferenceException as error:

                    print('Элемент с другой страницы')
                    print(error)
                    break

                except ElementClickInterceptedException as error:

                    print('Элемент перекрыт, прокручиваем вниз')
                    self.scroll_the_page_js(browser, h=50)


        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

#        content = {
#            'Ответы для выбора': [answers_to_choose_from, len(answers_to_choose_from)],
#            'Ответы для ввода обычные': [answers_for_entering_usual, len(answers_for_entering_usual)],
#            'Ответы для ввода чисел': [answers_for_entering_numbers, len(answers_for_entering_numbers)],
#            'Ответы для ввода большого текста': [answers_for_entering_large_text, len(answers_for_entering_large_text)],
#            'Ответы для выбора два': [answers_options, len(answers_options)],
#        }


        for name, answers in content_question_selenium.items():

            if name == 'Ответы для выбора' and content_question_selenium[name][0]:

                work_with_answers_to_choose_from(browser, answers[0])

            if name == 'Ответы для ввода обычные' and content_question_selenium[name][0]:

                work_with_answers_for_entering_usual(browser, answers[0])

            if name == 'Ответы для ввода чисел' and content_question_selenium[name][0]:

                work_with_answers_for_entering_number(browser, answers[0])

            if name == 'Ответы для выбора два' and content_question_selenium[name][0]:

                work_with_answers_to_choose_from_two(browser, answers[0])


    def work_with_question_or_finish(self, browser):

        """ Следующий вопрос или заканчиваем """

        def finish_test(browser):

            """ Заканчиваем тест? """

            well_what = browser.find_elements(
                By.XPATH,
                '//input[@placeholder="7 123 456 78-90"]'
            )

            if well_what: return 'Закончить'


        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

        while True:

            finish_it = finish_test(browser)
            if finish_it: return 'Закончить'

            question_name = self.get_question_name(browser)
            if not (question_name == 'Пусто') and question_name: return 'Работаем с вопросом'


    def wait_for_new_question(self, browser, old_question):

        """ Ждем новый вопрос """

        count = 0

        while True:

            if count == 4: break

            current_question = self.get_question_name(browser)

            if old_question != current_question: break
            else: print('Ждем новый вопрос')

            time.sleep(1)
            count += 1


    def get_content_question_done(self, question, content_question_selenium):

        """ Получаем содержимое вопроса в виде готового текста """

        def working_with_content(content, name, question):

            """ Работаем с содержимым """

            result = []

            if not len(content) or name == 'Ответы для выбора два': return []
            if question == 'Куда вам удобно приехать?': return []
            if 'Когда удобно заниматься?' in question: return []
            if 'Как зовут ученика' in question: return []
            if 'Когда нужна услуга?' in question: return []

            content = content[0]

            for element in content:

                try:

                    text = element.text
                    result.append(text)

                except StaleElementReferenceException as error:

                    continue

            return result


        def get_string_desired_type(result):

            """ Получаем строку нужного вида """

            result = '; '.join(result)
            index = result.index(';')
            result = result[:index] + result[index+1:]

            return result
        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

        result = [question]

        for name, content in content_question_selenium.items():

            questions = working_with_content(content, name, question)
            [result.append(question) for question in questions]

        result = [question for question in result if question]

        if len(result) == 1: result.append('Ответов нет, нужно указывать самому')

        string_desired_type = get_string_desired_type(result)

        return string_desired_type


    def take_test(self, browser):

        """ Проходим тест """

        def go_to_next_question(browser, old_question):

            """ Переходим к следующему вопросу """

            count = 0

            while True:

                current_question = self.get_question_name(browser)

                if old_question != current_question:

                    #print('Тут сразу все понятно')
                    break

                if old_question == current_question and int(count) == 1:

                    print('Нажиммаем сами')
                    self.click_continue(browser)
                    break

                time.sleep(.05)
                count += .1


        def show_output(content):

            """ Показываем вывод """

            for k, v in content.items():

                pass
                #print(f'{k} - {v}\n')


        def answer_certain_question(browser):

            """ Нужно ли отвечать на какие-либо определенные вопросы? """

            def try_to_find_map(browser):

                """ Пытаемся найти карту """

                map_selenium = browser.find_elements(
                    By.XPATH,
                    '//input[@placeholder="Город, улица и дом"]'
                )

                if map_selenium: return 'Есть карта'


            def try_to_find_calendar(browser):

                """ Пытаемся найти календарь """

                calendar_selenium = browser.find_elements(
                    By.XPATH,
                    '//table[1]//tbody/tr[last()]/td/div'
                )

                if calendar_selenium: return 'Есть календарь'


            # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

            well_what_map = try_to_find_map(browser)
            well_what_calendar = try_to_find_calendar(browser)

            return well_what_map, well_what_calendar


        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

        done_questions = []

        while True:

            well_what = self.work_with_question_or_finish(browser)    # Выходим?
            if well_what == 'Закончить':

                break

            question_name = self.get_question_name(browser)
            if question_name is None: continue

            content_question_selenium = self.get_content_question_selenium(browser, question_name)
            content_question_done = self.get_content_question_done(question_name, content_question_selenium)
            done_questions.append(content_question_done)

            well_what_map, well_what_calendar = answer_certain_question(browser)
            if well_what_map or well_what_calendar: self.answer_specific_question(browser, well_what_map, well_what_calendar)
            else: self.answer_question(browser, content_question_selenium)

            go_to_next_question(browser, question_name)
            self.wait_for_new_question(browser, question_name)

        return done_questions


    def write_data_to_google_table(self, questions, count):

        """ Записываем данные в google таблицу """

        letters = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'G', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z']
        gc = gspread.service_account(filename='service_account.json')
        sh = gc.open("profi")
        worksheet = sh.get_worksheet(0)

        for current_iteration, question in enumerate(questions):

            coordinate = f'{letters[current_iteration]}{count}'
            worksheet.update(coordinate, question)
            print(f'Записали - {question}')


    def work_on_test(self, list_links_to_page_with_test, city):

        """ Работаем над тестом """

        def open_test(browser):

            """ Открываем тест """

            go_to_test_button = browser.find_element(
                By.XPATH,
                '//span[contains(text(), "Найти специалиста")]/..'
            )
            self.click_on_button(go_to_test_button, .2, .5)

            while True:

                try:

                    go_to_test_button = browser.find_element(
                        By.XPATH,
                        '//span[contains(text(), "Найти специалиста")]/..'
                    )
                    self.click_on_button(go_to_test_button, .2, .5)

                except NoSuchElementException as error:

                    print('Кнопка "Найти специалиста" не найдена')
                    break


        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

        count = self.string_to_write
        list_links_to_page_with_test = list_links_to_page_with_test[self.test_number:]

        for link in list_links_to_page_with_test:  # Проходимся по страницам с тестом

            print(f'Текущий город - {city}\nТекущая страница с тестом - {link}')

            browser = self.open_page(link, fs='yes')    # Открываем страницу
            open_test(browser)                          # Открываем тест
            questions = self.take_test(browser)                     # Проходим тест

            self.write_data_to_google_table(questions, count)
            count += 1

            browser.quit()


    def work_on_links_to_tests(self, list_of_cities):

        """ Работаем над ссылками для теста  """

        def get_list_link_to_all_services(browser):

            """ Получаем список ссылок на все услуги """

            list_links_to_all_services = browser.find_elements(
                By.XPATH,
                '//div[@data-shmid]//span[contains(text(), "Все")]/ancestor::a'
            )
            list_links_to_all_services_done = [item.get_attribute('href') for item in list_links_to_all_services]

            return list_links_to_all_services_done


        def work_on_list_of_links_to_pages_with_text(browser, list_link_to_all_services, count):

            """ Работаем над получением списка услуг """

            def get_list_services_from_page_categories(browser):

                """ Получаем список сервисов со страницы категории """

                list_services_from_page_categories = browser.find_elements(
                    By.XPATH,
                    '//div[@class="services-catalog__content"]/a'
                )
                list_services_from_page_categories_done = [item.get_attribute('href') for item in list_services_from_page_categories]

                return list_services_from_page_categories_done


            def get_list_services_from_page_subcategories(browser):

                """ Получаем список сервисов со страницы подкатегории """

                list_services_from_page_subcategories_done = []
                list_services_from_page_subcategories = browser.find_elements(
                    By.XPATH,
                    '//div[@class="services-catalog__cols"]//a'
                )

                for current_iteration, link in enumerate(list_services_from_page_subcategories):

                    print(link.get_attribute('text'))
                    list_services_from_page_subcategories_done.append(link.get_attribute('href'))

                list_services_from_page_subcategories_done = [item.get_attribute('href') for item in list_services_from_page_subcategories]

                return list_services_from_page_subcategories_done


            def get_result(temp_result):

                """ Получаем результат """

                result = []

                for item in temp_result:

                    if re.findall(r'#', item): continue
                    result.append(item)

                return result

            # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

            result_list = []

            for link in list_link_to_all_services:

                if count == self.number_branches_to_pass_through: return result_list
                browser.get(link)

                list_services_from_page_categories = get_list_services_from_page_categories(browser)
                list_services_from_page_subcategories = get_list_services_from_page_subcategories(browser)
                temp_result = list_services_from_page_categories + list_services_from_page_subcategories
                result = get_result(temp_result)
                [result_list.append(link) for link in result]

                count += 1

            return result_list


        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

        count = 0
        list_of_cities = list_of_cities[self.city_number:]

        for city in list_of_cities:  # Проходимся по городам

            browser = self.open_page(city, fs='yes')
            list_link_to_all_services = get_list_link_to_all_services(browser)
            list_link_to_all_services = list_link_to_all_services[self.where_to_start:]
            list_links_to_page_with_test = work_on_list_of_links_to_pages_with_text(browser, list_link_to_all_services, count)
            browser.quit()

            self.work_on_test(list_links_to_page_with_test, city)


    def start_parsing(self):

        """ Начать парсинг """

        list_of_cities = self.get_list_of_cities()                                  # Ссылки на города
        self.work_on_links_to_tests(list_of_cities)
#        list_links_to_page_with_test = ['https://nnov.profi.ru/krasota/nogti']
#        self.work_on_test(list_links_to_page_with_test)
